import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/DNN_6h_TFM'
TIME_STEPS=36 #6h
UNITS=22
DROPOUT=0.779
ACTIVATION='tanh'
OPTIMIZER='adamax'
EPOCHS=85
BATCHSIZE=23
VALIDATIONSPLIT=0.2
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (3081, 36, 1) y_train shape: (3081,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(Dense(units=UNITS, input_shape=(X_train.shape[1], X_train.shape[2]), activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dropout(rate=DROPOUT))
model.add(Dense(X_train.shape[2],activation=ACTIVATION))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=[rmse,'mse'])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense (Dense) (None, 36, 22) 44 _________________________________________________________________ dense_1 (Dense) (None, 36, 16) 368 _________________________________________________________________ dropout (Dropout) (None, 36, 16) 0 _________________________________________________________________ dense_2 (Dense) (None, 36, 1) 17 ================================================================= Total params: 429 Trainable params: 429 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/85 108/108 [==============================] - 0s 3ms/step - loss: 0.7328 - rmse: 0.7723 - mse: 0.9363 - val_loss: 0.6838 - val_rmse: 0.6907 - val_mse: 0.6376 Epoch 2/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6690 - rmse: 0.7178 - mse: 0.7955 - val_loss: 0.5983 - val_rmse: 0.6100 - val_mse: 0.5232 Epoch 3/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6418 - rmse: 0.7007 - mse: 0.7455 - val_loss: 0.5508 - val_rmse: 0.5664 - val_mse: 0.4686 Epoch 4/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6282 - rmse: 0.6936 - mse: 0.7187 - val_loss: 0.5234 - val_rmse: 0.5417 - val_mse: 0.4401 Epoch 5/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6209 - rmse: 0.6912 - mse: 0.7052 - val_loss: 0.5039 - val_rmse: 0.5243 - val_mse: 0.4212 Epoch 6/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6139 - rmse: 0.6878 - mse: 0.6921 - val_loss: 0.4901 - val_rmse: 0.5119 - val_mse: 0.4085 Epoch 7/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6092 - rmse: 0.6853 - mse: 0.6843 - val_loss: 0.4802 - val_rmse: 0.5031 - val_mse: 0.3997 Epoch 8/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6041 - rmse: 0.6815 - mse: 0.6730 - val_loss: 0.4729 - val_rmse: 0.4966 - val_mse: 0.3935 Epoch 9/85 108/108 [==============================] - 0s 2ms/step - loss: 0.6022 - rmse: 0.6812 - mse: 0.6728 - val_loss: 0.4650 - val_rmse: 0.4896 - val_mse: 0.3868 Epoch 10/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5995 - rmse: 0.6792 - mse: 0.6693 - val_loss: 0.4595 - val_rmse: 0.4846 - val_mse: 0.3822 Epoch 11/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5971 - rmse: 0.6778 - mse: 0.6645 - val_loss: 0.4560 - val_rmse: 0.4815 - val_mse: 0.3794 Epoch 12/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5966 - rmse: 0.6778 - mse: 0.6652 - val_loss: 0.4524 - val_rmse: 0.4781 - val_mse: 0.3763 Epoch 13/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5935 - rmse: 0.6745 - mse: 0.6623 - val_loss: 0.4490 - val_rmse: 0.4750 - val_mse: 0.3735 Epoch 14/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5919 - rmse: 0.6735 - mse: 0.6590 - val_loss: 0.4467 - val_rmse: 0.4730 - val_mse: 0.3718 Epoch 15/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5898 - rmse: 0.6718 - mse: 0.6562 - val_loss: 0.4439 - val_rmse: 0.4704 - val_mse: 0.3695 Epoch 16/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5889 - rmse: 0.6712 - mse: 0.6553 - val_loss: 0.4411 - val_rmse: 0.4678 - val_mse: 0.3673 Epoch 17/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5869 - rmse: 0.6694 - mse: 0.6553 - val_loss: 0.4359 - val_rmse: 0.4629 - val_mse: 0.3631 Epoch 18/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5877 - rmse: 0.6712 - mse: 0.6576 - val_loss: 0.4318 - val_rmse: 0.4592 - val_mse: 0.3599 Epoch 19/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5866 - rmse: 0.6705 - mse: 0.6559 - val_loss: 0.4294 - val_rmse: 0.4571 - val_mse: 0.3582 Epoch 20/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5846 - rmse: 0.6683 - mse: 0.6529 - val_loss: 0.4269 - val_rmse: 0.4548 - val_mse: 0.3564 Epoch 21/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5847 - rmse: 0.6689 - mse: 0.6543 - val_loss: 0.4234 - val_rmse: 0.4515 - val_mse: 0.3536 Epoch 22/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5846 - rmse: 0.6685 - mse: 0.6528 - val_loss: 0.4225 - val_rmse: 0.4506 - val_mse: 0.3529 Epoch 23/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5826 - rmse: 0.6668 - mse: 0.6521 - val_loss: 0.4184 - val_rmse: 0.4467 - val_mse: 0.3498 Epoch 24/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5827 - rmse: 0.6675 - mse: 0.6554 - val_loss: 0.4149 - val_rmse: 0.4435 - val_mse: 0.3473 Epoch 25/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5826 - rmse: 0.6675 - mse: 0.6557 - val_loss: 0.4123 - val_rmse: 0.4411 - val_mse: 0.3454 Epoch 26/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5814 - rmse: 0.6669 - mse: 0.6545 - val_loss: 0.4100 - val_rmse: 0.4391 - val_mse: 0.3438 Epoch 27/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5805 - rmse: 0.6661 - mse: 0.6554 - val_loss: 0.4060 - val_rmse: 0.4354 - val_mse: 0.3410 Epoch 28/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5801 - rmse: 0.6656 - mse: 0.6566 - val_loss: 0.4039 - val_rmse: 0.4334 - val_mse: 0.3395 Epoch 29/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5808 - rmse: 0.6670 - mse: 0.6601 - val_loss: 0.4024 - val_rmse: 0.4320 - val_mse: 0.3385 Epoch 30/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5799 - rmse: 0.6663 - mse: 0.6567 - val_loss: 0.3986 - val_rmse: 0.4285 - val_mse: 0.3358 Epoch 31/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5802 - rmse: 0.6671 - mse: 0.6598 - val_loss: 0.3984 - val_rmse: 0.4284 - val_mse: 0.3358 Epoch 32/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5790 - rmse: 0.6656 - mse: 0.6562 - val_loss: 0.3968 - val_rmse: 0.4271 - val_mse: 0.3349 Epoch 33/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5789 - rmse: 0.6662 - mse: 0.6577 - val_loss: 0.3956 - val_rmse: 0.4260 - val_mse: 0.3343 Epoch 34/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5777 - rmse: 0.6651 - mse: 0.6554 - val_loss: 0.3925 - val_rmse: 0.4233 - val_mse: 0.3323 Epoch 35/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5795 - rmse: 0.6673 - mse: 0.6580 - val_loss: 0.3928 - val_rmse: 0.4236 - val_mse: 0.3327 Epoch 36/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5788 - rmse: 0.6661 - mse: 0.6571 - val_loss: 0.3909 - val_rmse: 0.4219 - val_mse: 0.3315 Epoch 37/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5769 - rmse: 0.6640 - mse: 0.6550 - val_loss: 0.3897 - val_rmse: 0.4209 - val_mse: 0.3308 Epoch 38/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5774 - rmse: 0.6649 - mse: 0.6587 - val_loss: 0.3865 - val_rmse: 0.4179 - val_mse: 0.3287 Epoch 39/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5764 - rmse: 0.6638 - mse: 0.6577 - val_loss: 0.3844 - val_rmse: 0.4162 - val_mse: 0.3277 Epoch 40/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5765 - rmse: 0.6644 - mse: 0.6561 - val_loss: 0.3819 - val_rmse: 0.4141 - val_mse: 0.3263 Epoch 41/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5751 - rmse: 0.6624 - mse: 0.6544 - val_loss: 0.3803 - val_rmse: 0.4126 - val_mse: 0.3253 Epoch 42/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5736 - rmse: 0.6604 - mse: 0.6506 - val_loss: 0.3793 - val_rmse: 0.4118 - val_mse: 0.3249 Epoch 43/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5705 - rmse: 0.6573 - mse: 0.6441 - val_loss: 0.3782 - val_rmse: 0.4110 - val_mse: 0.3244 Epoch 44/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5711 - rmse: 0.6568 - mse: 0.6440 - val_loss: 0.3771 - val_rmse: 0.4101 - val_mse: 0.3240 Epoch 45/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5696 - rmse: 0.6559 - mse: 0.6427 - val_loss: 0.3751 - val_rmse: 0.4084 - val_mse: 0.3229 Epoch 46/85 108/108 [==============================] - 0s 1ms/step - loss: 0.5684 - rmse: 0.6541 - mse: 0.6391 - val_loss: 0.3727 - val_rmse: 0.4065 - val_mse: 0.3218 Epoch 47/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5662 - rmse: 0.6526 - mse: 0.6356 - val_loss: 0.3725 - val_rmse: 0.4063 - val_mse: 0.3218 Epoch 48/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5646 - rmse: 0.6500 - mse: 0.6327 - val_loss: 0.3724 - val_rmse: 0.4063 - val_mse: 0.3218 Epoch 49/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5640 - rmse: 0.6490 - mse: 0.6291 - val_loss: 0.3717 - val_rmse: 0.4057 - val_mse: 0.3215 Epoch 50/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5638 - rmse: 0.6483 - mse: 0.6295 - val_loss: 0.3706 - val_rmse: 0.4046 - val_mse: 0.3207 Epoch 51/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5634 - rmse: 0.6487 - mse: 0.6271 - val_loss: 0.3701 - val_rmse: 0.4043 - val_mse: 0.3206 Epoch 52/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5613 - rmse: 0.6459 - mse: 0.6231 - val_loss: 0.3694 - val_rmse: 0.4037 - val_mse: 0.3203 Epoch 53/85 108/108 [==============================] - ETA: 0s - loss: 0.5869 - rmse: 0.6697 - mse: 0.68 - 0s 2ms/step - loss: 0.5612 - rmse: 0.6456 - mse: 0.6244 - val_loss: 0.3677 - val_rmse: 0.4021 - val_mse: 0.3192 Epoch 54/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5583 - rmse: 0.6423 - mse: 0.6183 - val_loss: 0.3665 - val_rmse: 0.4009 - val_mse: 0.3184 Epoch 55/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5595 - rmse: 0.6440 - mse: 0.6216 - val_loss: 0.3660 - val_rmse: 0.4004 - val_mse: 0.3181 Epoch 56/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5600 - rmse: 0.6450 - mse: 0.6235 - val_loss: 0.3667 - val_rmse: 0.4011 - val_mse: 0.3186 Epoch 57/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5587 - rmse: 0.6437 - mse: 0.6208 - val_loss: 0.3655 - val_rmse: 0.4000 - val_mse: 0.3179 Epoch 58/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5586 - rmse: 0.6438 - mse: 0.6211 - val_loss: 0.3630 - val_rmse: 0.3978 - val_mse: 0.3165 Epoch 59/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5592 - rmse: 0.6443 - mse: 0.6215 - val_loss: 0.3632 - val_rmse: 0.3979 - val_mse: 0.3166 Epoch 60/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5582 - rmse: 0.6435 - mse: 0.6196 - val_loss: 0.3621 - val_rmse: 0.3970 - val_mse: 0.3161 Epoch 61/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5585 - rmse: 0.6438 - mse: 0.6191 - val_loss: 0.3607 - val_rmse: 0.3957 - val_mse: 0.3153 Epoch 62/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5586 - rmse: 0.6437 - mse: 0.6228 - val_loss: 0.3586 - val_rmse: 0.3938 - val_mse: 0.3141 Epoch 63/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5567 - rmse: 0.6415 - mse: 0.6176 - val_loss: 0.3598 - val_rmse: 0.3948 - val_mse: 0.3147 Epoch 64/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5574 - rmse: 0.6424 - mse: 0.6209 - val_loss: 0.3587 - val_rmse: 0.3940 - val_mse: 0.3143 Epoch 65/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5589 - rmse: 0.6451 - mse: 0.6262 - val_loss: 0.3568 - val_rmse: 0.3922 - val_mse: 0.3132 Epoch 66/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5586 - rmse: 0.6446 - mse: 0.6234 - val_loss: 0.3577 - val_rmse: 0.3930 - val_mse: 0.3138 Epoch 67/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5578 - rmse: 0.6440 - mse: 0.6239 - val_loss: 0.3579 - val_rmse: 0.3933 - val_mse: 0.3139 Epoch 68/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5580 - rmse: 0.6440 - mse: 0.6234 - val_loss: 0.3559 - val_rmse: 0.3914 - val_mse: 0.3128 Epoch 69/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5568 - rmse: 0.6425 - mse: 0.6214 - val_loss: 0.3553 - val_rmse: 0.3909 - val_mse: 0.3125 Epoch 70/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5576 - rmse: 0.6439 - mse: 0.6202 - val_loss: 0.3535 - val_rmse: 0.3893 - val_mse: 0.3115 Epoch 71/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5562 - rmse: 0.6420 - mse: 0.6210 - val_loss: 0.3531 - val_rmse: 0.3890 - val_mse: 0.3113 Epoch 72/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5580 - rmse: 0.6441 - mse: 0.6238 - val_loss: 0.3518 - val_rmse: 0.3879 - val_mse: 0.3107 Epoch 73/85 108/108 [==============================] - 0s 3ms/step - loss: 0.5573 - rmse: 0.6439 - mse: 0.6243 - val_loss: 0.3525 - val_rmse: 0.3885 - val_mse: 0.3111 Epoch 74/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5576 - rmse: 0.6439 - mse: 0.6242 - val_loss: 0.3507 - val_rmse: 0.3869 - val_mse: 0.3101 Epoch 75/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5564 - rmse: 0.6425 - mse: 0.6237 - val_loss: 0.3506 - val_rmse: 0.3868 - val_mse: 0.3101 Epoch 76/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5580 - rmse: 0.6446 - mse: 0.6269 - val_loss: 0.3512 - val_rmse: 0.3873 - val_mse: 0.3104 Epoch 77/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5584 - rmse: 0.6454 - mse: 0.6277 - val_loss: 0.3514 - val_rmse: 0.3876 - val_mse: 0.3106 Epoch 78/85 108/108 [==============================] - 0s 3ms/step - loss: 0.5564 - rmse: 0.6428 - mse: 0.6217 - val_loss: 0.3504 - val_rmse: 0.3868 - val_mse: 0.3102 Epoch 79/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5576 - rmse: 0.6440 - mse: 0.6241 - val_loss: 0.3504 - val_rmse: 0.3866 - val_mse: 0.3099 Epoch 80/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5557 - rmse: 0.6419 - mse: 0.6214 - val_loss: 0.3503 - val_rmse: 0.3865 - val_mse: 0.3098 Epoch 81/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5579 - rmse: 0.6450 - mse: 0.6280 - val_loss: 0.3493 - val_rmse: 0.3856 - val_mse: 0.3093 Epoch 82/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5571 - rmse: 0.6439 - mse: 0.6263 - val_loss: 0.3484 - val_rmse: 0.3848 - val_mse: 0.3089 Epoch 83/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5565 - rmse: 0.6433 - mse: 0.6252 - val_loss: 0.3474 - val_rmse: 0.3839 - val_mse: 0.3083 Epoch 84/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5553 - rmse: 0.6413 - mse: 0.6224 - val_loss: 0.3485 - val_rmse: 0.3849 - val_mse: 0.3089 Epoch 85/85 108/108 [==============================] - 0s 2ms/step - loss: 0.5563 - rmse: 0.6421 - mse: 0.6255 - val_loss: 0.3470 - val_rmse: 0.3836 - val_mse: 0.3081
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.2498 Root Mean Square Error: 0.4458 Mean Square Error: 0.1987
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.95 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 790us/step - loss: 0.5825 - rmse: 0.6775 - mse: 0.9824 evaluate: [0.5825332999229431, 0.6775192022323608, 0.9823993444442749] LSTM:
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
Mean Absolute Error: 0.2261 Root Mean Square Error: 0.5791 Mean Square Error: 0.3354
anomalies: (98, 10)
###################################################### ####################### PM25 ###########################
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 873us/step - loss: 0.6014 - rmse: 0.6968 - mse: 0.9475 evaluate: [0.6013744473457336, 0.6968408226966858, 0.9474955797195435] LSTM: Mean Absolute Error: 0.2323 Root Mean Square Error: 0.5291 Mean Square Error: 0.2800
anomalies: (105, 10)
###################################################### ####################### PM10 ########################### Testing shape: (744, 36, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 2ms/step - loss: 0.6150 - rmse: 0.7112 - mse: 0.9302 evaluate: [0.6150347590446472, 0.7112275958061218, 0.9302341938018799] LSTM: Mean Absolute Error: 0.2402 Root Mean Square Error: 0.4866 Mean Square Error: 0.2368
anomalies: (66, 10)
###################################################### ####################### PM1ATM ###########################
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 2ms/step - loss: 0.6220 - rmse: 0.7223 - mse: 0.9484 evaluate: [0.6219671964645386, 0.7222810387611389, 0.9483834505081177] LSTM: Mean Absolute Error: 0.2380 Root Mean Square Error: 0.4946 Mean Square Error: 0.2446
anomalies: (94, 10)
###################################################### ####################### PM25ATM ###########################
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 2ms/step - loss: 0.6162 - rmse: 0.7155 - mse: 0.9516 evaluate: [0.6162117719650269, 0.7154954671859741, 0.95159912109375] LSTM: Mean Absolute Error: 0.2339 Root Mean Square Error: 0.5067 Mean Square Error: 0.2567
anomalies: (93, 10)
###################################################### ####################### PM10ATM ###########################
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
Testing shape: (744, 36, 1) 24/24 [==============================] - 0s 2ms/step - loss: 0.6108 - rmse: 0.7064 - mse: 0.9326 evaluate: [0.6107572913169861, 0.7063829302787781, 0.9325913190841675] LSTM: Mean Absolute Error: 0.2386 Root Mean Square Error: 0.5107 Mean Square Error: 0.2608
anomalies: (92, 10)
######################################################